{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-01-08T11:12:36.324855Z",
     "start_time": "2025-01-08T11:12:36.099164Z"
    }
   },
   "source": [
    "# 导入库\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 处理重复数据",
   "id": "450ca7b1c47bc6b2"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T11:23:01.378224Z",
     "start_time": "2025-01-08T11:23:01.371962Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 构建数据\n",
    "df_obj = pd.DataFrame({'data1': ['a'] * 4 + ['b'] * 4,\n",
    "                       'data2': np.random.randint(0, 4, 8)})\n",
    "\n",
    "# .duplicated() 方法可以检测重复数据,并返回重复数据的索引,返回值是一个布尔数组,True表示重复数据,False表示不重复数据\n",
    "print(f'重复数据索引: {df_obj.duplicated()}')\n",
    "\n",
    "# 取出不重复行\n",
    "# [~df_obj.duplicated()]表示取出不重复行的索引,返回值是一个Series\n",
    "# ~表示取反,即取出不重复行的索引\n",
    "print(f'不重复数据: \\n{df_obj.loc[~df_obj.duplicated()]}\\n')\n",
    "\n",
    "# 按照某一列去重\n",
    "# .drop_duplicated() 方法可以按照某一列去重,并返回去重后的数据\n",
    "print(f'去重后的数据: \\n{df_obj.duplicated(subset=\"data1\")}\\n')\n",
    "\n",
    "# 删除重复的行\n",
    "# .drop_duplicates() 方法可以删除重复的行,并返回去重后的数据\n",
    "print(f'删除重复的行后的数据: \\n{df_obj.drop_duplicates()}\\n')"
   ],
   "id": "6f08ab12e58a252",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "重复数据索引: 0    False\n",
      "1    False\n",
      "2     True\n",
      "3    False\n",
      "4    False\n",
      "5    False\n",
      "6     True\n",
      "7    False\n",
      "dtype: bool\n",
      "不重复数据: \n",
      "  data1  data2\n",
      "0     a      2\n",
      "1     a      0\n",
      "3     a      3\n",
      "4     b      3\n",
      "5     b      0\n",
      "7     b      2\n",
      "\n",
      "去重后的数据: \n",
      "0    False\n",
      "1     True\n",
      "2     True\n",
      "3     True\n",
      "4    False\n",
      "5     True\n",
      "6     True\n",
      "7     True\n",
      "dtype: bool\n",
      "\n",
      "删除重复的行后的数据: \n",
      "  data1  data2\n",
      "0     a      2\n",
      "1     a      0\n",
      "3     a      3\n",
      "4     b      3\n",
      "5     b      0\n",
      "7     b      2\n",
      "\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T11:23:02.826439Z",
     "start_time": "2025-01-08T11:23:02.821094Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df_obj1 = pd.DataFrame({'data1': [np.nan] * 4,\n",
    "                        'data2': list('1235')})\n",
    "print(f'原始数据: \\n{df_obj1}\\n')\n",
    "\n",
    "# 在pd的duplicated()中nan与nan相等\n",
    "print(f'重复数据索引: \\n{df_obj1.duplicated()}')\n",
    "\n"
   ],
   "id": "d27f00632c3054b9",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原始数据: \n",
      "   data1 data2\n",
      "0    NaN     1\n",
      "1    NaN     2\n",
      "2    NaN     3\n",
      "3    NaN     5\n",
      "\n",
      "重复数据索引: \n",
      "0    False\n",
      "1    False\n",
      "2    False\n",
      "3    False\n",
      "dtype: bool\n"
     ]
    }
   ],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T11:29:04.280063Z",
     "start_time": "2025-01-08T11:29:04.275035Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# map与applymap: 应用于所有元素\n",
    "# 1. map: 应用于Series的每个元素\n",
    "# 2. applymap: 应用于DataFrame的每个元素\n",
    "\n",
    "#异常值手动替换\n",
    "# .replace() 方法可以替换指定的值,并返回替换后的数据\n",
    "# 注意: replace() 方法可以替换多个值,但替换后的数据的顺序与替换前的顺序一致d\n",
    "ser_obj = pd.Series(np.arange(10), index=range(3, 13))\n",
    "print(f'原始数据: \\n{ser_obj}\\n')\n",
    "\n",
    "# 单个值替换单个值\n",
    "print(f'单个值替换单个值: \\n{ser_obj.replace(1, -100)}\\n')\n",
    "\n",
    "# 多个值替换单个值\n",
    "print(f'多个值替换单个值: \\n{ser_obj.replace([4, 7], -100)}\\n')\n",
    "\n",
    "# 多个值替换多个值\n",
    "print(f'多个值替换多个值: \\n{ser_obj.replace([4, 7], [-100, -200])}\\n')\n",
    "\n",
    "# 多个值替换一个值\n",
    "print(f'多个值替换一个值: \\n{ser_obj.replace(range(6, 9), -100)}\\n')"
   ],
   "id": "ea1b41d842bbbc4b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原始数据: \n",
      "3     0\n",
      "4     1\n",
      "5     2\n",
      "6     3\n",
      "7     4\n",
      "8     5\n",
      "9     6\n",
      "10    7\n",
      "11    8\n",
      "12    9\n",
      "dtype: int64\n",
      "\n",
      "单个值替换单个值: \n",
      "3       0\n",
      "4    -100\n",
      "5       2\n",
      "6       3\n",
      "7       4\n",
      "8       5\n",
      "9       6\n",
      "10      7\n",
      "11      8\n",
      "12      9\n",
      "dtype: int64\n",
      "\n",
      "多个值替换单个值: \n",
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7    -100\n",
      "8       5\n",
      "9       6\n",
      "10   -100\n",
      "11      8\n",
      "12      9\n",
      "dtype: int64\n",
      "\n",
      "多个值替换多个值: \n",
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7    -100\n",
      "8       5\n",
      "9       6\n",
      "10   -200\n",
      "11      8\n",
      "12      9\n",
      "dtype: int64\n",
      "\n",
      "多个值替换一个值: \n",
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7       4\n",
      "8       5\n",
      "9    -100\n",
      "10   -100\n",
      "11   -100\n",
      "12      9\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T11:30:06.890065Z",
     "start_time": "2025-01-08T11:30:06.885840Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 正则表达式替换\n",
    "# .str.replace() 方法可以用正则表达式替换字符串,并返回替换后的数据\n",
    "ser_obj = pd.Series(['apple', 'banana', 'orange', 'pear', 'grape'])\n",
    "print(f'原始数据: \\n{ser_obj}\\n')\n",
    "\n",
    "# 单个值替换单个值\n",
    "print(f'单个值替换单个值: \\n{ser_obj.str.replace(\"a\", \"A\")}\\n')\n",
    "\n",
    "# 多个值替换单个值\n",
    "print(f'多个值替换单个值: \\n{ser_obj.str.replace(\"a|e\", \"A\")}\\n')\n"
   ],
   "id": "71ecf77f794dd4b8",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原始数据: \n",
      "0     apple\n",
      "1    banana\n",
      "2    orange\n",
      "3      pear\n",
      "4     grape\n",
      "dtype: object\n",
      "\n",
      "单个值替换单个值: \n",
      "0     Apple\n",
      "1    bAnAnA\n",
      "2    orAnge\n",
      "3      peAr\n",
      "4     grApe\n",
      "dtype: object\n",
      "\n",
      "多个值替换单个值: \n",
      "0     apple\n",
      "1    banana\n",
      "2    orange\n",
      "3      pear\n",
      "4     grape\n",
      "dtype: object\n",
      "\n"
     ]
    }
   ],
   "execution_count": 19
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T11:30:10.236538Z",
     "start_time": "2025-01-08T11:30:10.228618Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 随机替换\n",
    "# .sample() 方法可以随机替换指定的值,并返回替换后的数据\n",
    "ser_obj = pd.Series(np.arange(10), index=range(3, 13))\n",
    "print(f'原始数据: \\n{ser_obj}\\n')\n",
    "\n",
    "# 单个值替换单个值\n",
    "print(f'单个值替换单个值: \\n{ser_obj.sample(n=1, random_state=42).values[0]}\\n')\n",
    "\n",
    "# 多个值替换单个值\n",
    "print(f'多个值替换单个值: \\n{ser_obj.sample(n=2, random_state=42).values}\\n')\n",
    "\n",
    "# 多个值替换多个值                                                                                                   \n",
    "print(f'多个值替换多个值: \\n{ser_obj.sample(n=2, random_state=42).values}\\n')\n",
    "\n",
    "# 多个值替换一个值\n",
    "print(f'多个值替换一个值: \\n{ser_obj.sample(n=3, random_state=42).values}\\n')"
   ],
   "id": "f626bc5d9a66f014",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原始数据: \n",
      "3     0\n",
      "4     1\n",
      "5     2\n",
      "6     3\n",
      "7     4\n",
      "8     5\n",
      "9     6\n",
      "10    7\n",
      "11    8\n",
      "12    9\n",
      "dtype: int64\n",
      "\n",
      "单个值替换单个值: \n",
      "8\n",
      "\n",
      "多个值替换单个值: \n",
      "[8 1]\n",
      "\n",
      "多个值替换多个值: \n",
      "[8 1]\n",
      "\n",
      "多个值替换一个值: \n",
      "[8 1 5]\n",
      "\n"
     ]
    }
   ],
   "execution_count": 20
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "59dfe085816b2cdf"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
